{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import albert\n",
    "import albert.modeling, albert.tokenization\n",
    "import transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"/home/husein/parsing/self-attentive-parser/src\")\n",
    "sys.path.append(\"/home/husein/parsing/self-attentive-parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import itertools\n",
    "import os.path\n",
    "import time\n",
    "import shutil\n",
    "import re\n",
    "import json\n",
    "\n",
    "import torch\n",
    "import torch.optim.lr_scheduler\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import evaluate\n",
    "import trees_newline as trees\n",
    "import vocabulary\n",
    "import nkutil\n",
    "import parse_nk_albert_base as parse_nk\n",
    "tokens = parse_nk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "if parse_nk.use_cuda:\n",
    "    info = torch.load('models/en_bert_dev=79.01.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = parse_nk.NKChartParser.from_spec(info['spec'], info['state_dict'])\n",
    "bert_model = info['spec']['hparams']['bert_model']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_treebank = trees.load_trees('test.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "tf.compat.v1.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "sd = parser.state_dict()\n",
    "\n",
    "LABEL_VOCAB = [x[0] for x in sorted(parser.label_vocab.indices.items(), key=lambda x: x[1])]\n",
    "TAG_VOCAB = [x[0] for x in sorted(parser.tag_vocab.indices.items(), key=lambda x: x[1])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "position_table = tf.constant(sd['embedding.position_table'].cpu().numpy(), name=\"position_table\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_layer_norm(input, torch_name, name):\n",
    "    # TODO(nikita): The epsilon here isn't quite the same as in pytorch\n",
    "    # The pytorch code adds eps=1e-3 to the standard deviation, while this\n",
    "    # tensorflow code adds eps=1e-6 to the variance.\n",
    "    # However, the resulting mismatch in floating-point values does not seem to\n",
    "    # translate to any noticable changes in the parser's tree output\n",
    "    mean, variance = tf.nn.moments(input, [1], keep_dims=True)\n",
    "    return tf.nn.batch_normalization(\n",
    "        input,\n",
    "        mean, variance,\n",
    "        offset=tf.constant(sd[f'{torch_name}.b_2'].cpu().numpy(), name=f\"{name}/offset\"),\n",
    "        scale=tf.constant(sd[f'{torch_name}.a_2'].cpu().numpy(), name=f\"{name}/scale\"),\n",
    "        variance_epsilon=1e-6)\n",
    "\n",
    "\n",
    "def make_heads(input, shape_bthf, shape_xtf, torch_name, name):\n",
    "    res = tf.matmul(input,\n",
    "        tf.constant(sd[torch_name].cpu().numpy().transpose((1,0,2)).reshape((512, -1)), name=f\"{name}/W\"))\n",
    "    res = tf.reshape(res, shape_bthf)\n",
    "    res = tf.transpose(res, (0,2,1,3)) # batch x num_heads x time x feat\n",
    "    res = tf.reshape(res, shape_xtf) # _ x time x feat\n",
    "    return res\n",
    "\n",
    "def make_attention(input, nonpad_ids, dim_flat, dim_padded, valid_mask, torch_name, name):\n",
    "    input_flat = tf.scatter_nd(indices=nonpad_ids[:, None], updates=input, shape=tf.concat([dim_flat, tf.shape(input)[1:]], axis=0))\n",
    "    input_flat_dat, input_flat_pos = tf.split(input_flat, 2, axis=-1)\n",
    "\n",
    "    shape_bthf = tf.concat([dim_padded, [8, -1]], axis=0)\n",
    "    shape_bhtf = tf.convert_to_tensor([dim_padded[0], 8, dim_padded[1], -1])\n",
    "    shape_xtf = tf.convert_to_tensor([dim_padded[0] * 8, dim_padded[1], -1])\n",
    "    shape_xf = tf.concat([dim_flat, [-1]], axis=0)\n",
    "\n",
    "    qs1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_qs1', f'{name}/q_dat')\n",
    "    ks1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_ks1', f'{name}/k_dat')\n",
    "    vs1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_vs1', f'{name}/v_dat')\n",
    "    qs2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_qs2', f'{name}/q_pos')\n",
    "    ks2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_ks2', f'{name}/k_pos')\n",
    "    vs2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_vs2', f'{name}/v_pos')\n",
    "\n",
    "    qs = tf.concat([qs1, qs2], axis=-1)\n",
    "    ks = tf.concat([ks1, ks2], axis=-1)\n",
    "    attn_logits = tf.matmul(qs, ks, transpose_b=True) / (1024 ** 0.5)\n",
    "\n",
    "    attn_mask = tf.reshape(tf.tile(valid_mask, [1,8*dim_padded[1]]), tf.shape(attn_logits))\n",
    "    # TODO(nikita): use tf.where and -float('inf') here?\n",
    "    attn_logits -= 1e10 * tf.to_float(~attn_mask)\n",
    "\n",
    "    attn = tf.nn.softmax(attn_logits)\n",
    "\n",
    "    attended_dat_raw = tf.matmul(attn, vs1)\n",
    "    attended_dat_flat = tf.reshape(tf.transpose(tf.reshape(attended_dat_raw, shape_bhtf), (0,2,1,3)), shape_xf)\n",
    "    attended_dat = tf.gather(attended_dat_flat, nonpad_ids)\n",
    "    attended_pos_raw = tf.matmul(attn, vs2)\n",
    "    attended_pos_flat = tf.reshape(tf.transpose(tf.reshape(attended_pos_raw, shape_bhtf), (0,2,1,3)), shape_xf)\n",
    "    attended_pos = tf.gather(attended_pos_flat, nonpad_ids)\n",
    "\n",
    "    out_dat = tf.matmul(attended_dat, tf.constant(sd[f'{torch_name}.proj1.weight'].cpu().numpy().transpose()))\n",
    "    out_pos = tf.matmul(attended_pos, tf.constant(sd[f'{torch_name}.proj2.weight'].cpu().numpy().transpose()))\n",
    "\n",
    "    out = tf.concat([out_dat, out_pos], -1)\n",
    "    return make_layer_norm(input + out, f'{torch_name}.layer_norm', f'{name}/layer_norm')\n",
    "\n",
    "def make_dense_relu_dense(input, torch_name, torch_type, name):\n",
    "    # TODO: use name\n",
    "    mul1 = tf.matmul(input, tf.constant(sd[f'{torch_name}.w_1{torch_type}.weight'].cpu().numpy().transpose()))\n",
    "    mul1b = tf.nn.bias_add(mul1, tf.constant(sd[f'{torch_name}.w_1{torch_type}.bias'].cpu().numpy()))\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, tf.constant(sd[f'{torch_name}.w_2{torch_type}.weight'].cpu().numpy().transpose()))\n",
    "    mul2b = tf.nn.bias_add(mul2, tf.constant(sd[f'{torch_name}.w_2{torch_type}.bias'].cpu().numpy()))\n",
    "    return mul2b\n",
    "\n",
    "def make_ff(input, torch_name, name):\n",
    "    # TODO: use name\n",
    "    input_dat, input_pos = tf.split(input, 2, axis=-1)\n",
    "    out_dat = make_dense_relu_dense(input_dat, torch_name, 'c', name=\"TODO_dat\")\n",
    "    out_pos = make_dense_relu_dense(input_pos, torch_name, 'p', name=\"TODO_pos\")\n",
    "    out = tf.concat([out_dat, out_pos], -1)\n",
    "    return make_layer_norm(input + out, f'{torch_name}.layer_norm', f'{name}/layer_norm')\n",
    "\n",
    "def make_stacks(input, nonpad_ids, dim_flat, dim_padded, valid_mask, num_stacks):\n",
    "    res = input\n",
    "    for i in range(num_stacks):\n",
    "        res = make_attention(res, nonpad_ids, dim_flat, dim_padded, valid_mask, f'encoder.attn_{i}', name=f'attn_{i}')\n",
    "        res = make_ff(res, f'encoder.ff_{i}', name=f'ff_{i}')\n",
    "    return res\n",
    "\n",
    "def make_layer_norm_with_constants(input, constants):\n",
    "    # TODO(nikita): The epsilon here isn't quite the same as in pytorch\n",
    "    # The pytorch code adds eps=1e-3 to the standard deviation, while this\n",
    "    # tensorflow code adds eps=1e-6 to the variance.\n",
    "    # However, the resulting mismatch in floating-point values does not seem to\n",
    "    # translate to any noticable changes in the parser's tree output\n",
    "    mean, variance = tf.nn.moments(input, [1], keep_dims=True)\n",
    "    return tf.nn.batch_normalization(\n",
    "        input,\n",
    "        mean, variance,\n",
    "        offset=constants[0],\n",
    "        scale=constants[1],\n",
    "        variance_epsilon=1e-6)\n",
    "\n",
    "def make_flabel_with_constants(input, constants):\n",
    "    mul1 = tf.matmul(input, constants[0])\n",
    "    mul1b = tf.nn.bias_add(mul1, constants[1])\n",
    "    mul1b = make_layer_norm_with_constants(mul1b, constants[2:4])\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, constants[4])\n",
    "    mul2b = tf.nn.bias_add(mul2, constants[5], name='flabel')\n",
    "    return mul2b\n",
    "\n",
    "def make_ftag(input):\n",
    "    constants = (\n",
    "        tf.constant(sd['f_tag.0.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_tag.0.bias'].cpu().numpy()),\n",
    "        tf.constant(sd['f_tag.1.b_2'].cpu().numpy(), name=\"tag/layer_norm/offset\"),\n",
    "        tf.constant(sd['f_tag.1.a_2'].cpu().numpy(), name=\"tag/layer_norm/scale\"),\n",
    "        tf.constant(sd['f_tag.3.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_tag.3.bias'].cpu().numpy()),\n",
    "    )\n",
    "    mul1 = tf.matmul(input, constants[0])\n",
    "    mul1b = tf.nn.bias_add(mul1, constants[1])\n",
    "    mul1b = make_layer_norm_with_constants(mul1b, constants[2:4])\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, constants[4])\n",
    "    mul2b = tf.nn.bias_add(mul2, constants[5], name='ftag')\n",
    "    return mul2b\n",
    "\n",
    "def make_flabel_constants():\n",
    "    return (\n",
    "        tf.constant(sd['f_label.0.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_label.0.bias'].cpu().numpy()),\n",
    "        tf.constant(sd['f_label.1.b_2'].cpu().numpy(), name=\"label/layer_norm/offset\"),\n",
    "        tf.constant(sd['f_label.1.a_2'].cpu().numpy(), name=\"label/layer_norm/scale\"),\n",
    "        tf.constant(sd['f_label.3.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_label.3.bias'].cpu().numpy()),\n",
    "    )\n",
    "\n",
    "def make_network():\n",
    "    # batch x num_subwords\n",
    "    input_ids = tf.placeholder(shape=(None, None), dtype=tf.int32, name='input_ids')\n",
    "    word_end_mask = tf.placeholder(shape=(None, None), dtype=tf.int32, name='word_end_mask')\n",
    "    input_dat, nonpad_ids, dim_flat, dim_padded, valid_mask, lengths = make_bert(input_ids, word_end_mask)\n",
    "    input_pos_flat = tf.tile(position_table[:dim_padded[1]], [dim_padded[0], 1])\n",
    "    input_pos = tf.gather(input_pos_flat, nonpad_ids)\n",
    "\n",
    "    input_joint = tf.concat([input_dat, input_pos], -1)\n",
    "    input_joint = make_layer_norm(input_joint, 'embedding.layer_norm', 'embedding/layer_norm')\n",
    "\n",
    "    word_out = make_stacks(input_joint, nonpad_ids, dim_flat, dim_padded, valid_mask, num_stacks=parser.spec['hparams']['num_layers'])\n",
    "    word_out = tf.concat([word_out[:, 0::2], word_out[:, 1::2]], -1)\n",
    "\n",
    "    # part-of-speech predictions\n",
    "    ftag = make_ftag(word_out)\n",
    "    tags_packed = tf.argmax(ftag, axis=-1)\n",
    "    tags = tf.reshape(\n",
    "        tf.scatter_nd(indices=nonpad_ids[:, None], updates=tags_packed, shape=dim_flat),\n",
    "        dim_padded\n",
    "        )\n",
    "    tags = tf.identity(tags, name=\"tags\")\n",
    "\n",
    "    fp_out = tf.concat([word_out[:-1,:512], word_out[1:,512:]], -1)\n",
    "\n",
    "    fp_start_idxs = tf.cumsum(lengths, exclusive=True)\n",
    "    fp_end_idxs = tf.cumsum(lengths) - 1 # the number of fenceposts is 1 less than the number of words\n",
    "\n",
    "    fp_end_idxs_uneven = fp_end_idxs - tf.convert_to_tensor([1, 0])\n",
    "\n",
    "    # Have to make these outside tf.map_fn for model compression to work\n",
    "    constants = make_flabel_constants()\n",
    "\n",
    "    def to_map(start_and_end):\n",
    "        start, end = start_and_end\n",
    "        fp = fp_out[start:end]\n",
    "        flabel = make_flabel_with_constants(tf.reshape(fp[None,:,:] - fp[:,None,:], (-1, 1024)), constants)\n",
    "        actual_chart_size = end-start\n",
    "        flabel = tf.reshape(flabel, [actual_chart_size, actual_chart_size, -1])\n",
    "        amount_to_pad = dim_padded[1] - actual_chart_size\n",
    "        # extra padding on the label dimension is for the not-a-constituent label,\n",
    "        # which always has a score of 0\n",
    "        flabel = tf.pad(flabel, [[0, amount_to_pad], [0, amount_to_pad], [1, 0]])\n",
    "        return flabel\n",
    "\n",
    "    charts = tf.map_fn(to_map, (fp_start_idxs, fp_end_idxs), dtype=(tf.float32))\n",
    "    charts = tf.identity(charts, name=\"charts\")\n",
    "\n",
    "    return input_ids, word_end_mask, charts, tags\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_bert(input_ids, word_end_mask):\n",
    "    # We can derive input_mask from either input_ids or word_end_mask\n",
    "    input_mask = (1 - tf.cumprod(1 - word_end_mask, axis=-1, reverse=True))\n",
    "    token_type_ids = tf.zeros_like(input_ids)\n",
    "    bert_model = make_bert_instance(input_ids, input_mask, token_type_ids)\n",
    "\n",
    "    bert_features = bert_model.get_sequence_output()\n",
    "    bert_features_packed = tf.gather(\n",
    "        tf.reshape(bert_features, [-1, int(bert_features.shape[-1])]),\n",
    "        tf.to_int32(tf.where(tf.reshape(word_end_mask, (-1,))))[:,0])\n",
    "    projected_annotations = tf.matmul(\n",
    "        bert_features_packed,\n",
    "        tf.constant(sd['project_bert.weight'].cpu().numpy().transpose()))\n",
    "\n",
    "    # input_mask is over subwords, whereas valid_mask is over words\n",
    "    sentence_lengths = tf.reduce_sum(word_end_mask, -1)\n",
    "    valid_mask = (tf.range(tf.reduce_max(sentence_lengths))[None,:] < sentence_lengths[:, None])\n",
    "    dim_padded = tf.shape(valid_mask)[:2]\n",
    "    mask_flat = tf.reshape(valid_mask, (-1,))\n",
    "    dim_flat = tf.shape(mask_flat)[:1]\n",
    "    nonpad_ids = tf.to_int32(tf.where(mask_flat)[:,0])\n",
    "\n",
    "    return projected_annotations, nonpad_ids, dim_flat, dim_padded, valid_mask, sentence_lengths\n",
    "\n",
    "def make_bert_instance(input_ids, input_mask, token_type_ids):\n",
    "    # Transfer BERT config into tensorflow implementation\n",
    "    config = albert.modeling.AlbertConfig.from_dict(parser.bert.config.to_dict())\n",
    "    config.hidden_act = 'gelu'\n",
    "    model = albert.modeling.AlbertModel(config=config, is_training=False,\n",
    "        input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids)\n",
    "\n",
    "    # Next, transfer learned weights (after fine-tuning)\n",
    "    bert_variables = [v for v in tf.get_collection('variables') if 'bert' in v.name]\n",
    "    tf.variables_initializer(bert_variables).run()\n",
    "    for variable in bert_variables:\n",
    "        name = variable.name.split(':')[0]\n",
    "\n",
    "        name = name.replace(\"module/\", \"\")\n",
    "\n",
    "        # Renaming and simplifying\n",
    "        name = name.replace(\"ffn_1\", \"ffn\")\n",
    "        # name = name.replace(\"bert/\", \"albert/\")\n",
    "        name = name.replace(\"attention_1\", \"attention\")\n",
    "        name = name.replace(\"transform/\", \"\")\n",
    "        name = name.replace(\"LayerNorm_1\", \"full_layer_layer_norm\")\n",
    "        name = name.replace(\"LayerNorm\", \"attention/LayerNorm\")\n",
    "        name = name.replace(\"transformer/\", \"\")\n",
    "\n",
    "        # The feed forward layer had an 'intermediate' step which has been abstracted away\n",
    "        name = name.replace(\"intermediate/dense/\", \"\")\n",
    "        name = name.replace(\"ffn/intermediate/output/dense/\", \"ffn_output/\")\n",
    "\n",
    "        # ALBERT attention was split between self and output which have been abstracted away\n",
    "        name = name.replace(\"/output/\", \"/\")\n",
    "        name = name.replace(\"/self/\", \"/\")\n",
    "\n",
    "        # The pooler is a linear layer\n",
    "        name = name.replace(\"pooler/dense\", \"pooler\")\n",
    "\n",
    "        # The classifier was simplified to predictions from cls/predictions\n",
    "        name = name.replace(\"cls/predictions\", \"predictions\")\n",
    "        name = name.replace(\"predictions/attention\", \"predictions\")\n",
    "\n",
    "        # Naming was changed to be more explicit\n",
    "        name = name.replace(\"embeddings/attention\", \"embeddings\")\n",
    "        name = name.replace(\"inner_group_\", \"albert_layers/\")\n",
    "        name = name.replace(\"group_\", \"albert_layer_groups/\")\n",
    "\n",
    "        # Classifier\n",
    "        if len(name.split(\"/\")) == 1 and (\"output_bias\" in name or \"output_weights\" in name):\n",
    "            name = \"classifier/\" + name\n",
    "\n",
    "        # No ALBERT model currently handles the next sentence prediction task\n",
    "        if \"seq_relationship\" in name:\n",
    "            name = name.replace(\"seq_relationship/output_\", \"sop_classifier/classifier/\")\n",
    "            name = name.replace(\"weights\", \"weight\")\n",
    "\n",
    "        name = name.split('/')\n",
    "        array = variable.eval()\n",
    "        # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v\n",
    "        # which are not required for using pretrained model\n",
    "        if any(n in [\"adam_v\", \"adam_m\"] for n in name):\n",
    "            print(\"Skipping {}\".format(\"/\".join(name)))\n",
    "            continue\n",
    "        pytorch_var = parser\n",
    "        for m_name in name:\n",
    "\n",
    "            if re.fullmatch(r'[A-Za-z]+_\\d+', m_name):\n",
    "                l = re.split(r'_(\\d+)', m_name)\n",
    "            else:\n",
    "                l = [m_name]\n",
    "            if l[0] == 'kernel' or l[0] == 'gamma':\n",
    "                pytorch_var = getattr(pytorch_var, 'weight')\n",
    "            elif l[0] == 'output_bias' or l[0] == 'beta':\n",
    "                pytorch_var = getattr(pytorch_var, 'bias')\n",
    "            elif l[0] == 'output_weights':\n",
    "                pytorch_var = getattr(pytorch_var, 'weight')\n",
    "            elif l[0] == 'cls':\n",
    "                pytorch_var = getattr(pytorch_var, 'cls')\n",
    "            else:\n",
    "                pytorch_var = getattr(pytorch_var, l[0])\n",
    "            if len(l) >= 2:\n",
    "                try:\n",
    "                    num = int(l[1])\n",
    "                    pytorch_var = pytorch_var[num]\n",
    "                except Exception as e:\n",
    "                    print(pytorch_var)\n",
    "                    raise\n",
    "        if m_name[-11:] == '_embeddings':\n",
    "            pytorch_var = getattr(pytorch_var, 'weight')\n",
    "        elif m_name == 'kernel':\n",
    "            pytorch_var = pytorch_var.t()\n",
    "        try:\n",
    "            assert pytorch_var.shape == array.shape\n",
    "        except AssertionError as e:\n",
    "            print(e)\n",
    "            e.args += (pytorch_var.shape, array.shape)\n",
    "            raise\n",
    "\n",
    "        variable.load(pytorch_var.detach().cpu().numpy())\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:194: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:507: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:588: The name tf.assert_less_equal is deprecated. Please use tf.compat.v1.assert_less_equal instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:1025: The name tf.AUTO_REUSE is deprecated. Please use tf.compat.v1.AUTO_REUSE instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:253: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:From <ipython-input-11-03800aaca47e>:120: Variable.load (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Prefer Variable.assign which has equivalent behavior in 2.X.\n",
      "WARNING:tensorflow:From <ipython-input-11-03800aaca47e>:10: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From <ipython-input-11-03800aaca47e>:10: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n",
      "WARNING:tensorflow:From <ipython-input-10-dc457fb7a117>:46: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    }
   ],
   "source": [
    "the_inp_tokens, the_inp_mask, the_out_chart, the_out_tags = make_network()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from parse_nk import BERT_TOKEN_MAPPING\n",
    "def bertify_batch(sentences):\n",
    "    all_input_ids = np.zeros((len(sentences), parser.bert_max_len), dtype=int)\n",
    "    all_word_end_mask = np.zeros((len(sentences), parser.bert_max_len), dtype=int)\n",
    "\n",
    "    subword_max_len = 0\n",
    "    for snum, sentence in enumerate(sentences):\n",
    "        tokens = []\n",
    "        word_end_mask = []\n",
    "\n",
    "        tokens.append(\"[CLS]\")\n",
    "        word_end_mask.append(1)\n",
    "\n",
    "        cleaned_words = []\n",
    "        for word in sentence:\n",
    "            word = BERT_TOKEN_MAPPING.get(word, word)\n",
    "            if word == \"n't\" and cleaned_words:\n",
    "                cleaned_words[-1] = cleaned_words[-1] + \"n\"\n",
    "                word = \"'t\"\n",
    "            cleaned_words.append(word)\n",
    "\n",
    "        for word in cleaned_words:\n",
    "            word_tokens = parser.bert_tokenizer.tokenize(word)\n",
    "            for _ in range(len(word_tokens)):\n",
    "                word_end_mask.append(0)\n",
    "            word_end_mask[-1] = 1\n",
    "            tokens.extend(word_tokens)\n",
    "        tokens.append(\"[SEP]\")\n",
    "        word_end_mask.append(1)\n",
    "\n",
    "        input_ids = parser.bert_tokenizer.convert_tokens_to_ids(tokens)\n",
    "\n",
    "        subword_max_len = max(subword_max_len, len(input_ids))\n",
    "\n",
    "        all_input_ids[snum, :len(input_ids)] = input_ids\n",
    "        all_word_end_mask[snum, :len(word_end_mask)] = word_end_mask\n",
    "\n",
    "    all_input_ids = all_input_ids[:, :subword_max_len]\n",
    "    all_word_end_mask = all_word_end_mask[:, :subword_max_len]\n",
    "    return all_input_ids, all_word_end_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 28/28 [00:02<00:00, 11.24it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "eval_batch_size = 16\n",
    "test_predicted = []\n",
    "for start_index in tqdm(range(0, len(test_treebank), eval_batch_size)):\n",
    "    subbatch_trees = test_treebank[start_index:start_index+eval_batch_size]\n",
    "    subbatch_sentences = [[leaf.word for leaf in tree.leaves()] for tree in subbatch_trees]\n",
    "    inp_val_tokens, inp_val_mask = bertify_batch([[word for word in sentence] for sentence in subbatch_sentences])\n",
    "    out_val_chart, out_val_tags = sess.run((the_out_chart, the_out_tags), \n",
    "                                       {the_inp_tokens: inp_val_tokens, the_inp_mask: inp_val_mask})\n",
    "    trees = []\n",
    "    scores = []\n",
    "    for snum, sentence in enumerate(subbatch_sentences):\n",
    "        chart_size = len(sentence) + 1\n",
    "        tf_chart = out_val_chart[snum,:chart_size,:chart_size,:]\n",
    "        sentence = list(zip([TAG_VOCAB[idx] for idx in out_val_tags[snum,1:chart_size]], [x for x in sentence]))\n",
    "        tree, score = parser.decode_from_chart(sentence, tf_chart)\n",
    "        trees.append(tree)\n",
    "        scores.append(score)\n",
    "    test_predicted.extend([p.convert() for p in trees])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<trees_newline.InternalTreebankNode object at 0x7f5d24aeb9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41746a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2423f4a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4177278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24240d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4177d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24243668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4179898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24245f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc417e400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24248828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc417edd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2424c128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc417f7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2424da90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41822b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d242525c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4182f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24256208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4184be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24257dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41868d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2425c9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc418b550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24260588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc418e1d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24264208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc418ee10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24265dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4191b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2426b828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41947f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2426f470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41eeda0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24271630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41eeef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241f6320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41712b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241f6d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4199a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241f95c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4199f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241fa4e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41ecb38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241fd400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc419a240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241fe320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc419a908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24200240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc419af98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24203128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc419e668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24203fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc419ed30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24204ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41a1470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24206e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41a1b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2420b320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41a51d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2420be10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41a5ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2420d908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41a7048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2420f400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41a7ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2420fef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41a7e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d242119e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc419d2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24214550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc419d6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24214fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc419dba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24215ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc419df98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24218b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41ab4a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2421cba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41af208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24221cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41afdd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24224e80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4133c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24228080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4135550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2422a278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4135e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2422b668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4138748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2422c9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc413b080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2422fd68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc413b9b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24232160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc413e278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241b4588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41dbac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241b8668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41442b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241bd320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4143048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241bf390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc414a2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241c2438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc414a908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241c44e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc414af28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241c65c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc414c588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241c75f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc414cba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241ca668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc414e208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241cb748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc414e860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241cd828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc414eef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241d2438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41525c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241d7080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41562b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241d7828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4156e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241d99b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4157208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241d9f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc415b198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241dc5f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41dcb00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241dcba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41df160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241de198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41df940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241de780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41df828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241dedd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41df2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241df438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41df240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241dfa20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41df5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241e6160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41dfe10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241e9550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41e3908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241ec080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41e5cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241ecba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc413ed68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241ed6d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41db0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241ef208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41db748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241efcf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc415b588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241f1828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc415b940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241f3358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc415bdd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241f3e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40f2710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24177588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40f2be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24177e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40f55f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24179748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40f59b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2417a0b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40f5d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2417a9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40f81d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2417c2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40f8588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2417cc18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40f8940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2417e588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40f8c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2417eeb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40fb080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241817f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40fb438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24185fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40fbef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2418c828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41010b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2418ff28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4103278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24196748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41073c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2419af60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc410a630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241a1748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc411acf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241a4f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc411de10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241a86d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41218d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241aada0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4123358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241ae5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4123dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241b1cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4126828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24135550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc412a2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24136c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc412ad68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2413b438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc412d828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2413cc50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41302b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24144748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4130d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2414a208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40b5160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2414ecf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40ba550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24154748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40bd940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2415b278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40bfd68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2415ed68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40c41d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24161c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc416e2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24164b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc416e940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241659e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4234358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241688d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc410cd30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2416a7b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41d8208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2416c6a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41d8780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2416e588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4112d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24170470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4112630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24173da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4115198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240f66d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4115eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240f9fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41188d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240fd898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40c9b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24101240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40cc5f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24103b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40ccfd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24108518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40d0a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2410ae80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40d3588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2410b518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4161438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2410bb70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4161c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2410e208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4161cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2410e828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4166128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2410ee48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41661d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241114a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41667b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24111b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4166518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24112198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4166908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24116358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4166a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24117eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc416a828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2411d5f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40d6780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24122cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40db128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24126c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40ddac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d241291d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40df7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24129710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40df9e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24129c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40dfbe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2412b198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40e8550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2412b6a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40e8748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2412bbe0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40e8908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2412c128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40e8b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2412c668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40e8cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24133128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40e8eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240b6080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40ef1d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240b7048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40ef8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240b7ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4116080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240bae10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4116780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240bcda0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4116e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240becf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40715c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240c0c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4071cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240c2b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4074400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240c7400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4074b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240c8d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40764e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240cd710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc408e550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240d2240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4091080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240d3ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4091a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240d75f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc408f438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240dc0b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc408fef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240dda20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40979e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240e2f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40994a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240e5908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc409d550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240ea2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40a11d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240ecc88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40a1dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240f06a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc409f9b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24075080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40a45c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24077a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40aa208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2407c438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40aae48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2407de10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40a6a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24084390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40ad5f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24087860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4038550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2408ce10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc403e7b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24092390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40409e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24096898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc403cb00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2409ae80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4044c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240a1470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4046e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240a5a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc404f080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240a9048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4052240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240ac6a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4052b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240aec88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4055550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240b2198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4055f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240b37f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4057908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24036dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc405b320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2403a3c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc405bcc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2403c9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc405f6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240410b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40610b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24041e80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87bb208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24043c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87bb7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24044a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87bbe10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24047828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87be438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24049668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87bea58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2404b4a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87c1080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2404d2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87c16d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240529e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87c1cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2405c160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87c7550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24061828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87cae10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24067f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87d1668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24071780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87d2fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ff5e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87db828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fff588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87e20f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24004cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87e5a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2400a908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87eb320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24011518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc408ab70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d240180f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4036c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2401dc88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc406dd68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24024898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87f0cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d2402a470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87f40b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d24030048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87f7470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fb4c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb877a7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fb8780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb877ec88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fb92e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87810f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fb9dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8781550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fba908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87819b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fbd4a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8781e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fbdfd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8784278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fc0b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87846a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fc16a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8784b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fc6320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8784f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fc7f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb879c160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fcdba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb879ceb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fd0860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87a0c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fd5518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87a3940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fda0f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87a66d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fdbdd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87a8438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fe0a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87ab1d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fe46d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87abeb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fe95f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87afc50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fec4e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87b1a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ff2da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87b57b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f7c588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb873b128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f81dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb873ea58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f8a5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87454a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f91da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8748eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f995c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb874e978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fa0d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb878e9b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fa9588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8793470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23faada0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87564a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23fb0710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb875b080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f37048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb875d438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f3b9b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb875f780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f42240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8763b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f44b38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8766ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f4b4e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb876c2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f4f860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb876e6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f50be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8773048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f52f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8773908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f56358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8776240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f59518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8776ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f5a710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87783c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f5d860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8778cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f5e9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8706a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f60ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8709358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f64d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8709be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f6ae80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb870c4e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f73438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87111d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23efaa20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8714ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23efff60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb871a470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f084e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb871dda0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f0fb70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8723780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f18240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb872a0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f1b898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb872d9b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f22cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8730b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f25d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb87364e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f2b5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86ba320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f2ce80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86baef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23f30748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86bdc18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23eb5048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc407a320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23eb7898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4084b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ebc128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40872b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ebd9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40873c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ec12b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc406c128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ec4780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb878e400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ec6d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86fc6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23eca240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8701048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ecb710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8701c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ecec18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86c2198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ed3128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86c2ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ed45c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86c5588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ed7b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86c5fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23edbac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86c4a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23edfa90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86c9860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ee2a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86cb6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ee7a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86d7780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23eeba20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86d4400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23eef9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86da080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e74908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86dacc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e78908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86de978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e7d780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86e0780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e7fd30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86e34a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e81fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86e3eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e84da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86e6748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e8a710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86e8588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e8e400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86e8f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e93128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86ecb00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e94dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86ef6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e98a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86f12e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e9e7b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86f1e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ea14e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86f5a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ea4cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41616a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ea8518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86ceeb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23eaad68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc40384a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23eb0550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb867f160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23eb1d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb867fc50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e36550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8681748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e38dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8684240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e3c5f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8684d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e3eb38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8687898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e430b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb868b278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e45668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb868bcc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e47c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb868e6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e4c198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8691080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e4e710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8691ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e4fd30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86934e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e53390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8693e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e57630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86a07f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e5c8d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86a2748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e5fb70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86a5748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e64da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86aa748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e67fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86ac828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e6d160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86af860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e70f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86affd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23df4470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86b4588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23df7978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86b4f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23df9e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86b7908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23dfd2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8639320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23dff780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8639c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e01c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb863c6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e06240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb863f0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e077b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb863fac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e0b908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8642470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e0fac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb869ba90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e14be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8645748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e17cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb864b278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e1be10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb864bf28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e20f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8647d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e27080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8650b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e2b240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8653ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e2d5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86559b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e2f978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86582e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23e30d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8658ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23db50f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb865b4a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23db74a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb865bd68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23db9828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb865f6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23dbbba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb865ff28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23dbdf28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8662860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23dc2978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8664198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23dc5a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85fb2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23dc9cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85fe358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23dcef60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8600400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23dd5240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86043c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23dd9550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86073c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ddd6d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8609438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23de1978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb860c4e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23de5c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb860f518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23dec390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8613588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23defef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86165f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d76588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86187f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d7ac50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb861b978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d802e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8620b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d849e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8622c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d8a080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8625e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d8f748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8628f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d91ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86d7240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d96048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb866e048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d974e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb866e8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d9a908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8671320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d9cd30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8671cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23da01d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8673898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23da3588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8673f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23da59e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8676828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23da6e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85f92b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23daa978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85f9da0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23db1278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86999b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d34b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86319e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d3b0b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8633a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d3e278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8636a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d3fdd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85ba208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d44320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85bacf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d44b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41ce588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d46320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc4076f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d46b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41d97b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d48320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41d94a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d48b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41d9dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d492b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41d9240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d49a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41dceb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d4d2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41dc7b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d51d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41dc518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d55908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85c4748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d58cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85c7390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d5c8d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85c7d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d62550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85cab70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d65160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85cc978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d67da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85cf828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d6ca20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85d2668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d705f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85e3a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cf5278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85e68d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cf6eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85ea780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cfd080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85ec630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d010f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85ef630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d05240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85f25f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d0a2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85f55f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d0d2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8579518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d13400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb857b518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d16588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb857f4a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d18828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85824a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d1bc18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8582eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d200b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85c0898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d21470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85882e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d237b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8588cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d26ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb858b748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d28f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85dd4e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d2c2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85d70f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23d30ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85d77f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cb65f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85e1240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cb7da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb858d390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cbc588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb858dcc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cbecc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8592860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cc24a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8596588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cc4c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85980f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cc94a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb8598c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ccac88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb859b8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ccc860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb859d550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ccf400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb859dac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23ccff60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85a1080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cd1b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85a1588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cd26d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb85a1ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cd42b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41cea58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cd4dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cc41ce160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f5d23cd69b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f5cb86a0208>\n"
     ]
    }
   ],
   "source": [
    "test_fscore = evaluate.evalb('EVALB/', test_treebank[:len(test_predicted)], test_predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'(Recall=77.57, Precision=80.50, FScore=79.01, CompleteMatch=5.77, TaggingAccuracy=90.30)'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(test_fscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_node_names = [the_inp_tokens.name.split(':')[0], the_inp_mask.name.split(':')[0]]\n",
    "output_node_names = [the_out_chart.name.split(':')[0], the_out_tags.name.split(':')[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-18-f034bd79213f>:1: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 23 variables.\n",
      "INFO:tensorflow:Converted 23 variables to const ops.\n"
     ]
    }
   ],
   "source": [
    "graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, output_node_names)\n",
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "graph_def = TransformGraph(graph_def, input_node_names, output_node_names, [\n",
    "'strip_unused_nodes()',\n",
    "'remove_nodes(op=Identity, op=CheckNumerics)',\n",
    "'fold_constants()',\n",
    "'fold_old_batch_norms',\n",
    "'fold_batch_norms',\n",
    "'round_weights(num_steps=128)',\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('export/albert-base.pb', 'wb') as f:\n",
    "    f.write(graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('vocab-albert-base.json', 'w') as fopen:\n",
    "    json.dump({'label': LABEL_VOCAB, 'tag': TAG_VOCAB}, fopen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
